{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from galileo.experiments import get_experiment, get_experiments\n",
    "from galileo.projects import get_project\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv(\"../.env\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_name = \"gpt-4.1-nano-2025-04-14\"\n",
    "\n",
    "project_id = get_project(name=project_name).id\n",
    "# experiment = get_experiment(project_id=project_id, experiment_name=experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt-4.1-mini-2025-04-14-banking-extreme_scenario_recovery\n",
      "gpt-4.1-mini-2025-04-14-banking-empathetic_resolution\n",
      "gpt-4.1-mini-2025-04-14-banking-scope_management\n",
      "gpt-4.1-mini-2025-04-14-banking-adaptive_tool_use\n",
      "gpt-4.1-nano-2025-04-14-banking-adversarial_input_mitigation\n",
      "gpt-4.1-nano-2025-04-14-banking-extreme_scenario_recovery\n",
      "gpt-4.1-nano-2025-04-14-banking-empathetic_resolution\n",
      "gpt-4.1-nano-2025-04-14-banking-scope_management\n",
      "gpt-4.1-nano-2025-04-14-banking-adaptive_tool_use\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_name</th>\n",
       "      <th>average_tool_selection_quality</th>\n",
       "      <th>model</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4.1-mini-2025-04-14-banking-extreme_scenar...</td>\n",
       "      <td>0.887500</td>\n",
       "      <td>gpt-4.1-mini-2025-04-14</td>\n",
       "      <td>extreme_scenario_recovery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4.1-mini-2025-04-14-banking-empathetic_res...</td>\n",
       "      <td>0.876335</td>\n",
       "      <td>gpt-4.1-mini-2025-04-14</td>\n",
       "      <td>empathetic_resolution</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-4.1-mini-2025-04-14-banking-scope_management</td>\n",
       "      <td>0.817000</td>\n",
       "      <td>gpt-4.1-mini-2025-04-14</td>\n",
       "      <td>scope_management</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4.1-mini-2025-04-14-banking-adaptive_tool_use</td>\n",
       "      <td>0.883838</td>\n",
       "      <td>gpt-4.1-mini-2025-04-14</td>\n",
       "      <td>adaptive_tool_use</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gpt-4.1-nano-2025-04-14-banking-adversarial_in...</td>\n",
       "      <td>0.853935</td>\n",
       "      <td>gpt-4.1-nano-2025-04-14</td>\n",
       "      <td>adversarial_input_mitigation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gpt-4.1-nano-2025-04-14-banking-extreme_scenar...</td>\n",
       "      <td>0.574379</td>\n",
       "      <td>gpt-4.1-nano-2025-04-14</td>\n",
       "      <td>extreme_scenario_recovery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gpt-4.1-nano-2025-04-14-banking-empathetic_res...</td>\n",
       "      <td>0.678250</td>\n",
       "      <td>gpt-4.1-nano-2025-04-14</td>\n",
       "      <td>empathetic_resolution</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gpt-4.1-nano-2025-04-14-banking-scope_management</td>\n",
       "      <td>0.742581</td>\n",
       "      <td>gpt-4.1-nano-2025-04-14</td>\n",
       "      <td>scope_management</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>gpt-4.1-nano-2025-04-14-banking-adaptive_tool_use</td>\n",
       "      <td>0.754834</td>\n",
       "      <td>gpt-4.1-nano-2025-04-14</td>\n",
       "      <td>adaptive_tool_use</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     experiment_name  \\\n",
       "0  gpt-4.1-mini-2025-04-14-banking-extreme_scenar...   \n",
       "1  gpt-4.1-mini-2025-04-14-banking-empathetic_res...   \n",
       "2   gpt-4.1-mini-2025-04-14-banking-scope_management   \n",
       "3  gpt-4.1-mini-2025-04-14-banking-adaptive_tool_use   \n",
       "4  gpt-4.1-nano-2025-04-14-banking-adversarial_in...   \n",
       "5  gpt-4.1-nano-2025-04-14-banking-extreme_scenar...   \n",
       "6  gpt-4.1-nano-2025-04-14-banking-empathetic_res...   \n",
       "7   gpt-4.1-nano-2025-04-14-banking-scope_management   \n",
       "8  gpt-4.1-nano-2025-04-14-banking-adaptive_tool_use   \n",
       "\n",
       "   average_tool_selection_quality                    model  \\\n",
       "0                        0.887500  gpt-4.1-mini-2025-04-14   \n",
       "1                        0.876335  gpt-4.1-mini-2025-04-14   \n",
       "2                        0.817000  gpt-4.1-mini-2025-04-14   \n",
       "3                        0.883838  gpt-4.1-mini-2025-04-14   \n",
       "4                        0.853935  gpt-4.1-nano-2025-04-14   \n",
       "5                        0.574379  gpt-4.1-nano-2025-04-14   \n",
       "6                        0.678250  gpt-4.1-nano-2025-04-14   \n",
       "7                        0.742581  gpt-4.1-nano-2025-04-14   \n",
       "8                        0.754834  gpt-4.1-nano-2025-04-14   \n",
       "\n",
       "                       category  \n",
       "0     extreme_scenario_recovery  \n",
       "1         empathetic_resolution  \n",
       "2              scope_management  \n",
       "3             adaptive_tool_use  \n",
       "4  adversarial_input_mitigation  \n",
       "5     extreme_scenario_recovery  \n",
       "6         empathetic_resolution  \n",
       "7              scope_management  \n",
       "8             adaptive_tool_use  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a pandas dataframe with the experiment name, average tool selection quality, and average action completion\n",
    "\n",
    "data = []\n",
    "for exp in get_experiments(project_id=project_id):\n",
    "    if exp.name == \"gpt-4.1-mini-2025-04-14-banking-adversarial_input_mitigation\":\n",
    "        continue\n",
    "    print(exp.name)\n",
    "    model, category = exp.name.split(\"-banking-\")\n",
    "    metrics = exp.additional_properties['aggregate_metrics']        \n",
    "    data.append({'experiment_name': exp.name, 'average_tool_selection_quality': metrics['average_tool_selection_quality'], 'model': model, 'category': category})\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "gpt-4.1-mini-2025-04-14    0.866168\n",
       "gpt-4.1-nano-2025-04-14    0.720796\n",
       "Name: average_tool_selection_quality, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get avg average_tool_selection_quality group by model and category\n",
    "df.groupby(['model'])['average_tool_selection_quality'].mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (al)",
   "language": "python",
   "name": "al"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
